import os
from tqdm import tqdm
import requests
import json
import pandas as pd

data_path = "../source_data/knowledge_base/"

url = 'http://113.31.111.86:19003/knowledge1/insert_data/'

# df_custom = pd.read_excel(data_path+"custom_qa.xlsx")

# for idx in tqdm(range(0,df_custom.shape[0],20)):
#     if idx + 20 <= df_custom.shape[0]:
#         data_ = {
#         "text":json.dumps([df_custom.iloc[k]['text'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "meta_data":json.dumps([df_custom.iloc[k]['meta'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "detail_text":json.dumps([df_custom.iloc[k]['detail_text'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "source":json.dumps([df_custom.iloc[k]['source_name'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "base":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "label":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "embedding":json.dumps([df_custom.iloc[k]['embedding_name'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "base_type":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+20)],ensure_ascii=False),
#         "project":json.dumps([df_custom.iloc[k]['project_name']+"_new4" for k in range(idx, idx+20)],ensure_ascii=False),
#         "single_data":0}
#     else:
#         tmp_len = df_custom.shape[0] - idx
#         data_ = {
#             "text":json.dumps([df_custom.iloc[k]['text'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "meta_data":json.dumps([df_custom.iloc[k]['meta'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "detail_text":json.dumps([df_custom.iloc[k]['detail_text'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "source":json.dumps([df_custom.iloc[k]['source_name'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "base":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "label":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "embedding":json.dumps([df_custom.iloc[k]['embedding_name'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "base_type":json.dumps([df_custom.iloc[k]['base_name'] for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "project":json.dumps([df_custom.iloc[k]['project_name']+"_new4" for k in range(idx, idx+tmp_len)],ensure_ascii=False),
#             "single_data":0}
#     res = requests.post(url=url, data=data_)
#     print(res.json())


# # 导入法律数据
# dir_list_ = ['法律法条','法律书籍','法律文书模版','法律案例','法律考试','日常法律问答']
# dir_list = ['legal_articles','legal_books','legal_templates','legal_cases','judicialExamination','legal_QA']
# dir_dic = dict(zip(dir_list,dir_list_))

# for item_ in dir_list:
#     data = []
#     dir_path = data_path + item_
#     for filename in os.listdir(dir_path):
#         if filename.endswith('.json'):
#             with open(os.path.join(dir_path,filename),"r") as f:
#                 if 'legal_articles' in item_:
#                     lines = f.readlines()
#                     for line in lines:
#                         line_data = json.loads(line)
#                         line_ = {"question":line_data['key'],"answer":line_data['value'],"metadata":{"source":item_+' '+filename,"label":dir_dic[item_]}}
#                         data.append(line_)
#                 else:
#                     data_ = json.load(f)
#                     data_ = [{"question":key,"answer":value,"metadata":{"source":item_+' '+filename,"label":dir_dic[item_]}} for key,value in data_.items()]
#                     data.extend(data_)
                    
    
#     pro_qa = 'legal_articles_qa_new4'
#     qa = item_ + '_qa'
#     tmp = data
#     num = 200
#     for idx in tqdm(range(0,len(tmp),num), desc=item_):
#         if idx + num <= len(tmp):
#             data_ = {
#                 "text":json.dumps([item['question'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "base":json.dumps([qa]*num),
#                 "label":json.dumps([qa]*num),
#                 "embedding":json.dumps(["text2vec_large"]*num),
#                 "base_type":json.dumps([qa]*num),
#                 "project":json.dumps([pro_qa]*num),
#                 "single_data":0,
#                 "is_normalize": "false"
#             }
#         else:
#             tmp_len = len(tmp) - idx
#             data_ = {
#                 "text":json.dumps([item['question'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+num]],ensure_ascii=False),
#                 "base":json.dumps([qa]*tmp_len),
#                 "label":json.dumps([qa]*tmp_len),
#                 "embedding":json.dumps(["text2vec_large"]*tmp_len),
#                 "base_type":json.dumps([qa]*tmp_len),
#                 "project":json.dumps([pro_qa]*tmp_len),
#                 "single_data":0,
#                 "is_normalize": "false"
#             }
#         res = requests.post(url=url, data=data_)
#         print(res.json())
        
# 导入其他库         
base2project = {
    "park_faq": "park_qa",
    "park_qa": "park_qa",
}

# 园区数据

with open(data_path+"build_base/script/park/faq.json", "r") as f:
    tmp = json.load(f)
for idx in tqdm(range(0,len(tmp),20)):
    if idx + 20 <= len(tmp):
        data_ = {
            "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "base":json.dumps(["park_faq"]*20),
            "label":json.dumps(["park_faq"]*20),
            "embedding":json.dumps(["m3e-base"]*20),
            "base_type":json.dumps(["park_faq"]*20),
            "project":json.dumps([base2project["park_faq"]]*20),
        }
    else:
        tmp_len = len(tmp) - idx
        data_ = {
            "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "base":json.dumps(["park_faq"]*tmp_len),
            "label":json.dumps(["park_faq"]*tmp_len),
            "embedding":json.dumps(["m3e-base"]*tmp_len),
            "base_type":json.dumps(["park_faq"]*tmp_len),
            "project":json.dumps([base2project["park_faq"]]*tmp_len),
        }
    res = requests.post(url=url, data=data_)
    print(res.json())
    
    
# with open(data_path+"build_base/script/park/faiss.json", "r") as f:
#     tmp = json.load(f)
# for idx in tqdm(range(0,len(tmp),20)):
#     if idx + 20 <= len(tmp):
#         data_ = {
#             "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "detail_text":json.dumps(["无" for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "base":json.dumps(["park_qa"]*20),
#             "label":json.dumps(["park_qa"]*20),
#             "embedding":json.dumps(["m3e-base"]*20),
#             "base_type":json.dumps(["park_qa"]*20),
#             "project":json.dumps([base2project["park_qa"]]*20),
#         }
#     else:
#         tmp_len = len(tmp) - idx
#         data_ = {
#             "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "detail_text":json.dumps(["无" for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#             "base":json.dumps(["park_qa"]*tmp_len),
#             "label":json.dumps(["park_qa"]*tmp_len),
#             "embedding":json.dumps(["m3e-base"]*tmp_len),
#             "base_type":json.dumps(["park_qa"]*tmp_len),
#             "project":json.dumps([base2project["park_qa"]]*tmp_len),
#         }
#     res = requests.post(url=url, data=data_)
#     print(res.json())
        

# 保险数据
# with open(data_path+"build_base/cache/insurance/faq.json", "r") as f:
#     data = json.load(f)
# tmp = list(data.values())

# for idx in tqdm(range(0,len(tmp),20)):
#     if idx + 20 <= len(tmp):
#         data_ = {
#         "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "base":json.dumps(["insurance_faq"]*20),
#         "label":json.dumps(["insurance_faq"]*20),
#         "embedding":json.dumps(["m3e-base"]*20),
#         "base_type":json.dumps(["insurance_faq"]*20),
#         "project":json.dumps(["insurance_qa_new4"]*20),
#         "single_data":0}
#     else:
#         tmp_len = len(tmp) - idx
#         data_ = {
#         "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "base":json.dumps(["insurance_faq"]*tmp_len),
#         "label":json.dumps(["insurance_faq"]*tmp_len),
#         "embedding":json.dumps(["m3e-base"]*tmp_len),
#         "base_type":json.dumps(["insurance_faq"]*tmp_len),
#         "project":json.dumps(["insurance_qa"]*tmp_len),
#         "single_data":0}
#     res = requests.post(url=url, data=data_)
#     print(res.json())
    
# with open(data_path+"build_base/cache/insurance/faiss.json", "r") as f:
#     data = json.load(f)
# tmp = list(data.values())

# for idx in tqdm(range(0,len(tmp),20)):
#     if idx + 20 <= len(tmp):
#         data_ = {
#         "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "detail_text":json.dumps(['无' for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "base":json.dumps(["insurance_qa"]*20),
#         "label":json.dumps(["insurance_qa"]*20),
#         "embedding":json.dumps(["m3e-base"]*20),
#         "base_type":json.dumps(["insurance_qa"]*20),
#         "project":json.dumps(["insurance_qa_new4"]*20),
#         "single_data":0}
#     else:
#         tmp_len = len(tmp) - idx
#         data_ = {
#         "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "meta_data":json.dumps([item['metadata'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "detail_text":json.dumps(['无' for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "source":json.dumps([item['metadata']['source'] for item in tmp[idx:idx+20]],ensure_ascii=False),
#         "base":json.dumps(["insurance_qa"]*tmp_len),
#         "label":json.dumps(["insurance_qa"]*tmp_len),
#         "embedding":json.dumps(["m3e-base"]*tmp_len),
#         "base_type":json.dumps(["insurance_qa"]*tmp_len),
#         "project":json.dumps(["insurance_qa"]*tmp_len),
#         "single_data":0}
#     res = requests.post(url=url, data=data_)
#     print(res.json())

# unidt数据
with open(data_path+"build_base/cache/unichat/faiss.json",'r') as f:
    data = json.load(f)
    
tmp = list(data.values())

for idx in tqdm(range(0,len(tmp),20)):
    if idx + 20 <= len(tmp):
        data_ = {
        "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
        "meta_data":json.dumps([{"source":"unidt_faq"}]*20,ensure_ascii=False),
        "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
        "source":json.dumps(["unidt_faq.txt"]*20,ensure_ascii=False),
        "base":json.dumps(["unidt_faq"]*20),
        "label":json.dumps(["unidt_faq"]*20),
        "embedding":json.dumps(["m3e-base"]*20),
        "base_type":json.dumps(["unidt_faq"]*20),
        "project":json.dumps(["unidt_qa"]*20),
        "single_data":0}
    else:
        tmp_len = len(tmp) - idx
        data_ = {
        "text":json.dumps([item['question'] for item in tmp[idx:idx+20]],ensure_ascii=False),
        "meta_data":json.dumps([{"source":"unidt_faq"}]*tmp_len,ensure_ascii=False),
        "detail_text":json.dumps([item['answer'] for item in tmp[idx:idx+20]],ensure_ascii=False),
        "source":json.dumps(["unidt_faq.txt"]*tmp_len,ensure_ascii=False),
        "base":json.dumps(["unidt_faq"]*tmp_len),
        "label":json.dumps(["unidt_faq"]*tmp_len),
        "embedding":json.dumps(["m3e-base"]*tmp_len),
        "base_type":json.dumps(["unidt_faq"]*tmp_len),
        "project":json.dumps(["unidt_qa"]*tmp_len),
        "single_data":0}
    res = requests.post(url=url, data=data_)
    print(res.json())
    

for idx in tqdm(range(0,len(tmp),20)):
    if idx + 20 <= len(tmp):
            data_ = {
            "text":json.dumps([item["answer"] for item in tmp[idx:idx+20]],ensure_ascii=False),
            "meta_data":json.dumps([{"source":"unidt_qa"}]*20,ensure_ascii=False),
            "detail_text":json.dumps(['无']*20,ensure_ascii=False),
            "source":json.dumps(["unidt_qa.txt"]*20,ensure_ascii=False),
            "base":json.dumps(["unidt_qa"]*20),
            "label":json.dumps(["unidt_qa"]*20),
            "embedding":json.dumps(["m3e-base"]*20),
            "base_type":json.dumps(["unidt_qa"]*20),
            "project":json.dumps(["unidt_qa"]*20),
            "single_data":0}
    else:
            tmp_len = len(tmp) - idx
            data_ = {
            "text":json.dumps([item["answer"] for item in tmp[idx:]],ensure_ascii=False),
            "meta_data":json.dumps([{"source":"unidt_qa"}]*tmp_len,ensure_ascii=False),
            "detail_text":json.dumps(['无']*tmp_len,ensure_ascii=False),
            "source":json.dumps(["unidt_qa.txt"]*tmp_len,ensure_ascii=False),
            "base":json.dumps(["unidt_qa"]*tmp_len),
            "label":json.dumps(["unidt_qa"]*tmp_len),
            "embedding":json.dumps(["m3e-base"]*tmp_len),
            "base_type":json.dumps(["unidt_qa"]*tmp_len),
            "project":json.dumps(["unidt_qa"]*tmp_len),
            "single_data":0}
    res = requests.post(url=url, data=data_)
    print(res.json())
        
